package com.evision.webcrawler.baidu;


import com.evision.webcrawler.baidu.parser.BaiduJsonParser;
import com.evision.webcrawler.baidu.url.BaiduUrlProducer;

import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;

/**
 * @author xiang
 *         百度图像单机爬虫
 */
public class BaiduCrawler {
    private ExecutorService executor;                //服务线程池

    private int ejNums;                                //Baidu图像Json接口,请求执行线程数
    private BlockingQueue<RequestUrl> urlQueue;        //Baidu图像Json接口,请求url队列
    private int urlQueuelength;                        //Baidu图像Json接口,请求url队列长度

    private int eiNums;                                //图像下载执行线程数
    private BlockingQueue<RequestUrl> imageQueue;    //图像下载请求url队列
    private int imageQueueLength;                    //图像下载请求url队列长度

    private int total;                                //每个关键词默认采集图像数量
    private String savePath;                        //保存从路径
    private int thinkingTime;                        //baidu接口请求thinking time
    private List<String> wordList;                   //关键词列表

    /**
     * 百度爬虫
     *
     * @param ejn          Baidu图像Json接口,请求执行线程数
     * @param ein          图像下载执行线程数
     * @param uql          Baidu图像Json接口,请求url队列长度
     * @param iql          图像下载请求url队列长度
     * @param savePath     下载图像保存路径
     * @param thinkingTime baidu接口请求thinking time
     */
    public BaiduCrawler(int ejn, int ein, int uql, int iql, int total, String savePath,
                        int thinkingTime,List wordList) {
        this.executor = Executors.newFixedThreadPool(ejn + ein + 1);
        this.urlQueuelength = uql;
        this.imageQueueLength = iql;
        this.ejNums = ejn;
        this.eiNums = ein;
        this.urlQueue = new LinkedBlockingQueue<RequestUrl>(urlQueuelength);
        this.imageQueue = new LinkedBlockingQueue<RequestUrl>(imageQueueLength);
        this.savePath = savePath;
        this.thinkingTime = thinkingTime;
        this.total = total;
        this.wordList = wordList;
    }


    /**
     * 开始执行爬取任务
     */
    public void crawlerRun() {
        ArrayList<CrawlerCell> bjl = new ArrayList<CrawlerCell>();
        ArrayList<CrawlerCell> bil = new ArrayList<CrawlerCell>();

        //baidu 接口url生成线程
        BaiduUrlProducer bp = new BaiduUrlProducer(urlQueue, total, wordList);

        BlockingQueue<RequestUrl>[] bql = new BlockingQueue[1];
        bql[0] = imageQueue;
        //baidu url请求线程
        for (int i = 0; i < ejNums; i++) {
            bjl.add(new CrawlerCell(executor, urlQueue, bql,
                    BaiduJsonParser.getInstance(savePath), this.thinkingTime));
        }

//        //图像下载线程
//        for (int j = 0; j < eiNums; j++) {
//            bil.add(new CrawlerCell(executor, imageQueue, null,
//                    BaiduImageDowloader.getInstance(savePath), 0));
//        }

        //执行线程
        this.executor.execute(bp);

        for (int i = 0; i < ejNums; i++) {
            this.executor.execute(bjl.get(i));
        }

//        for (int j = 0; j < eiNums; j++) {
//            this.executor.execute(bil.get(j));
//        }
        executor.shutdown();
    }

    public static void main(String[] args) {
        List list = new ArrayList();
        list.add("酒水饮料");
        list.add("结婚酒水 饮料 搭配");
        list.add("宴席酒水");
        list.add("宴会酒水点心");
        list.add("自助餐酒水");
        list.add("宴席酒水");
        list.add("酒店外建筑");

        String savepath = "";
        List workList = new ArrayList();
        for (int i=0; i< list.size() ;i++) {
            savepath = "/home/boyun/img/"+list.get(i) + File.separator;
            workList.add(list.get(i));
            File file = new File(savepath);
            if (!file.exists()){
                file.mkdirs();
            }
        }
        BaiduCrawler baiduCrawler = new BaiduCrawler(4,4,10,10,1000,savepath,2000,workList);
        baiduCrawler.crawlerRun();
    }
}
